# 1998 - 2016
########################################################
year = NIPA_6.8D$Year
agr = NIPA_6.8D$`        Agriculture, forestry, fishing, and hunting`
ind = NIPA_6.8D$Mining + NIPA_6.8D$Construction + NIPA_6.8D$Manufacturing + NIPA_6.8D$Utilities
srv = NIPA_6.8D$`Persons engaged in production1` - ind - agr
total = NIPA_6.8D$`Persons engaged in production1`
error = ((agr + ind + srv) - total)  / total
d6 = data.table(year, agr, ind, srv, total, source = "d6")
result = rbind(d1, d2, d3, d4, d5, d6)
# splice randomly
########################################################
year.start = sample(result$year, 1)
back = result[year <= year.start]$year
forward = result[year >= year.start]$year
source('~/Desktop/service/data/US_historical/workbook.R')
sources = c("d1", "d2", "d3", "d4", "d5", "d6")
l = length(year.ba)
l = length(year.back)
i = 1
year.sub = year.back[c(i, i + 1)]
l = length(year.back) -1
year.sub = year.back[c(i, i + 1)]
year.back = result[year <= year.start]$year %>% sort(., decreasing = T)
year.forward = result[year >= year.start]$year
start.sub = result[year == year.start]
start.sub = start.sub[ sample(nrow(start.sub), 1)   ]
sources = c("d1", "d2", "d3", "d4", "d5", "d6")
l = length(year.back) -1
year.sub = year.back[c(i, i + 1)]
year.back
year.back = result[year <= year.start]$year %>% sort(., decreasing = T) %>% unique()
year.forward = result[year >= year.start]$year %>% unique()
year.start = sample(result$year, 1)
year.back = result[year <= year.start]$year %>% sort(., decreasing = T) %>% unique()
year.forward = result[year >= year.start]$year %>% unique()
year.sub = year.back[c(i, i + 1)]
year.back
sub = result[year == year.sub]
sub = result[year %in% year.sub]
View(sub)
sample(unique(sub$source))
sample(unique(sub$source), 1)
source.sample = sample(unique(sub$source), 1)
sub = sub[source = source.sample]
source.sample = sample(unique(sub$source), 1)
sub = sub[source == source.sample]
View(sub)
gr = sub[1, 2:4]
gr = sub[1, 2:5]
gr = sub[1, 2:5] / sub[2, 2:5]
View(sub)
gr
output = start.sub
output
new = output[1,2:5] * gr
new
output = start.sub[2:5]
new = output[1] * gr
output = start.sub[2:5]
new = output[1] * gr
output
output = start.sub[,2:5]
new = output[1] * gr
output = rbind(new, output)
for(i in 1:l){
year.sub = year.back[c(i, i + 1)]
sub = result[year %in% year.sub]
source.sample = sample(unique(sub$source), 1)
sub = sub[source == source.sample]
gr = sub[1, 2:5] / sub[2, 2:5]
new = output[1] * gr
output = rbind(new, output)
}
View(output)
output = start.sub[,1:5]
year.sub
year.back
i
l
l = length(year.back) -1
l
l = length(year.back) -1
# back
l = length(year.back) -1
for(i in 1:l){
year.sub = year.back[c(i, i + 1)]
sub = result[year %in% year.sub]
source.sample = sample(unique(sub$source), 1)
sub = sub[source == source.sample]
gr = sub[1, 2:5] / sub[2, 2:5]
new = output[1] * gr
output = rbind(new, output)
}
year.sub
# back
l = length(year.back) -1
for(i in 1:l){
year.sub = year.back[c(i, i + 1)]
sub = result[year %in% year.sub]
source.sample = sample(unique(sub$source), 1)
sub = sub[source == source.sample]
gr = sub[1, 2:5] / sub[2, 2:5]
new = output[1] * gr
output = rbind(new, output)
}
output = start.sub[,2:5]
l = length(year.back) -1
for(i in 1:l){
year.sub = year.back[c(i, i + 1)]
sub = result[year %in% year.sub]
source.sample = sample(unique(sub$source), 1)
sub = sub[source == source.sample]
gr = sub[1, 2:5] / sub[2, 2:5]
new = output[1] * gr
output = rbind(new, output)
}
View(output)
year.sub = year.back[c(i, i + 1)]
sub = result[year %in% year.sub]
source.sample = sample(unique(sub$source), 1)
sub = sub[source == source.sample]
View(sub)
gr = sub[1, 2:5] / sub[2, 2:5]
new = output[1] * gr
output = rbind(new, output)
View(output)
new
year.sub = year.back[c(i, i + 1)]
sub = result[year %in% year.sub]
source.sample = sample(unique(sub$source), 1)
sub = sub[source == source.sample]
View(sub)
gr = sub[1, 2:5] / sub[2, 2:5]
gr
# back
l = length(year.back) -1
for(i in 1:l){
year.sub = year.back[c(i, i + 1)]
sub = result[year %in% year.sub]
source.sample = sample(unique(sub$source), 1)
sub = sub[source == source.sample]
gr = sub[1, 2:5] / sub[2, 2:5]
new = output[1] * gr
output = rbind(new, output)
}
View(output)
d1
d1 = d1[year <= 1920]
d1
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
View(output)
source('~/Desktop/service/data/US_historical/workbook.R')
output = cbind(year.back, output)
View(output)
output = cbind(sort(year.back), output)
View(output)
for(i in 1:l){
year.sub = year.back[c(i, i + 1)]
sub = result[year %in% year.sub]
source.sample = sample(unique(sub$source), 1)
sub = sub[source == source.sample]
gr = sub[1, 2:5] / sub[2, 2:5]
new = output[1] * gr
output = rbind(new, output)
if(is.na(output[1,1])) print(i)
}
source('~/Desktop/service/data/US_historical/workbook.R')
i = 50
year.sub = year.back[c(i, i + 1)]
sub = result[year %in% year.sub]
source.sample = sample(unique(sub$source), 1)
sub = sub[source == source.sample]
View(sub)
source.sample = sample(unique(sub$source), 1)
sub = sub[source == source.sample]
gr = sub[1, 2:5] / sub[2, 2:5]
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
View(output)
if(is.na(gr[1,1])) print(i, source.sample)
for(i in 1:l){
year.sub = year.back[c(i, i + 1)]
sub = result[year %in% year.sub]
source.sample = sample(unique(sub$source), 1)
sub = sub[source == source.sample]
gr = sub[1, 2:5] / sub[2, 2:5]
new = output[1] * gr
output = rbind(new, output)
if(is.na(gr[1,1])) print(i, source.sample)
}
source('~/Desktop/service/data/US_historical/workbook.R')
for(i in 1:l){
year.sub = year.back[c(i, i + 1)]
sub = result[year %in% year.sub]
source.sample = sample(unique(sub$source), 1)
sub = sub[source == source.sample]
gr = sub[1, 2:5] / sub[2, 2:5]
new = output[1] * gr
output = rbind(new, output)
if(is.na(gr[1,1])) {
print(i)
print(sub)
}
}
library(data.table)
library(here)
library(magrittr)
dir = here()
wd = paste(dir, "/Raw", sep = "")
setwd(wd)
files = list.files()
names = gsub(".csv","", files)
for(i in 1:length(files)){
temp = fread(files[i])
assign(names[i], temp)
}
# 1800 - 1920
#############################################################
# extrapolate missing values
extrapolate = function(x.in, y.in, x.out){
x = x.in
y = log(y.in)
r = lm(y ~ x)
x = data.table(x = x.out)
p = exp(predict(r, x)) %>% round()
return(p)
}
# construction
x.in = BA814_830$Year[1:10]
y.in = BA814_830$Construction[1:10]
x.out = seq(1800, 1830, 10)
BA814_830$Construction[1:4] = extrapolate(x.in, y.in, x.out )
# manufacturing
x.in = BA814_830$Year[1:10]
y.in = BA814_830$Manufacturing[1:10]
x.out = c(1800, 1820, 1830)
BA814_830$Manufacturing[c(1, 3,4)] = extrapolate(x.in, y.in, x.out )
# aggregate
agr = BA814_830$Agriculture + BA814_830$Fishing
ind = BA814_830$Mining +BA814_830$Construction + BA814_830$Manufacturing
srv = BA814_830$Total - agr - ind
total = BA814_830$Total
year = BA814_830$Year
error = ((agr + ind + srv) - total)  / total
d1 = data.table(year, agr, ind, srv, total, source = "d1")
d1 = d1[year <= 1920]
# 1920 - 1929
###########################################################
# aggregate
year.agr =  BA470_477$Year
agr = BA470_477$Farm
ind = BA840_848$Mining + BA840_848$Construction + BA840_848$Manufacturing
srv = BA840_848$Total - ind
year.srv = BA840_848$Year
year = intersect(year.agr, year.srv)
agr = agr[year.agr %in% year]
ind = ind[year.srv %in% year]
srv = srv[year.srv %in% year]
total = BA470_477$Employed[year.agr %in% year]
error = ((agr + ind + srv) - total)  / total
d2 = data.table(year, agr, ind, srv, total, source = "d2")
# 1929 - 1948
###########################################################
year = NIPA_6.8A$Year
agr = NIPA_6.8A$Farms + NIPA_6.8A$`Agricultural services, forestry, and fisheries `
ind = NIPA_6.8A$Mining + NIPA_6.8A$`Contract construction` + NIPA_6.8A$`Durable goods` + NIPA_6.8A$`Nondurable goods` +
NIPA_6.8A$`Gas, electric, and sanitary services `
srv = NIPA_6.8A$Total - agr - ind
total = NIPA_6.8A$Total
error = ((agr + ind + srv) - total)  / total
d3 = data.table(year, agr, ind, srv, total, source = "d3")
# 1948 - 1987
########################################################
year = NIPA_6.8B$year
agr = NIPA_6.8B$`Agriculture forestry and fishing`
ind = NIPA_6.8B$Mining + NIPA_6.8B$Construction + NIPA_6.8B$Manufacturing + NIPA_6.8B$`Electric gas and sanitary services`
srv = NIPA_6.8B$`Persons engaged in production` - agr - ind
total = NIPA_6.8B$`Persons engaged in production`
error = ((agr + ind + srv) - total)  / total
d4 = data.table(year, agr, ind, srv, total, source = "d4")
# 1987 - 2000
########################################################
year = NIPA_6.8C$Year
agr = NIPA_6.8C$`    Agriculture, forestry, and fishing`
ind = NIPA_6.8C$Mining + NIPA_6.8C$Construction + NIPA_6.8C$Manufacturing + NIPA_6.8C$`      Electric, gas, and sanitary services`
srv = NIPA_6.8C$`Persons engaged in production` - agr - ind
total = NIPA_6.8C$`Persons engaged in production`
error = ((agr + ind + srv) - total)  / total
d5 = data.table(year, agr, ind, srv, total, source =  "d5")
# 1998 - 2016
########################################################
year = NIPA_6.8D$Year
agr = NIPA_6.8D$`        Agriculture, forestry, fishing, and hunting`
ind = NIPA_6.8D$Mining + NIPA_6.8D$Construction + NIPA_6.8D$Manufacturing + NIPA_6.8D$Utilities
srv = NIPA_6.8D$`Persons engaged in production1` - ind - agr
total = NIPA_6.8D$`Persons engaged in production1`
error = ((agr + ind + srv) - total)  / total
d6 = data.table(year, agr, ind, srv, total, source = "d6")
result = rbind(d1, d2, d3, d4, d5, d6)
View(d2)
d2 = d2[year >= 1920]
source('~/Desktop/service/data/US_historical/workbook.R')
View(output)
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
View(result)
source('~/Desktop/service/data/US_historical/workbook.R')
source.sample = sample(unique(sub$source), 1)
table(sub$source)
n = data.table(table(sub$source))
View(n)
n = n[N > 1]
n
source.sample = sample(unique(sub$source), 1)
n = n[N > 1]
View(n)
source.sample = sample(n$V1, 1)
year.sub = year.back[c(i, i + 1)]
sub = result[year %in% year.sub]
n = data.table(table(sub$source))
n = n[N > 1]
source.sample = sample(n$V1, 1)
sub = sub[source == source.sample]
gr = sub[1, 2:5] / sub[2, 2:5]
new = output[1] * gr
source('~/Desktop/service/data/US_historical/workbook.R')
year.sub = year.back[c(i, i + 1)]
sub = result[year %in% year.sub]
n = data.table(table(sub$source))
n = n[N > 1]
View(n)
View(sub)
d2 = d2[year >= 1920]
source('~/Desktop/service/data/US_historical/workbook.R')
View(output)
output = cbind(year = sort(year.back), output)
source('~/Desktop/service/data/US_historical/workbook.R')
plot(output$year, output$agr
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
plot(output$year, output$agr, xlim = c(1800, 2016))
lines(output$year, output$agr)
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
l = length(year.forward) -1
year.sub = year.forward[c(i, i + 1)]
sub = result[year %in% year.sub]
n = data.table(table(sub$source))
year.sub = year.forward[c(i, i + 1)]
i = 1
year.sub = year.forward[c(i, i + 1)]
sub = result[year %in% year.sub]
n = data.table(table(sub$source))
n = n[N > 1]
source.sample = sample(n$V1, 1)
sub = sub[source == source.sample]
gr = sub[1, 2:5] / sub[2, 2:5]
gr
View(sub)
gr = sub[2, 2:5] / sub[1, 2:5]
gr
View(sub)
new = output[1] * gr
year.sub = year.forward[c(i, i + 1)]
sub = result[year %in% year.sub]
n = data.table(table(sub$source))
n = n[N > 1]
source.sample = sample(n$V1, 1)
sub = sub[source == source.sample]
gr = sub[2, 2:5] / sub[1, 2:5]
new = output[1] * gr
source('~/Desktop/service/data/US_historical/workbook.R')
output.forward = cbind(year = year.forward, output)
View(output)
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
View(output.back)
View(output.forward)
source('~/Desktop/service/data/US_historical/workbook.R')
plot(final$total)
View(final)
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
View(final)
source('~/Desktop/service/data/US_historical/workbook.R')
gr
new = tail(output,1) * gr
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
source('~/Desktop/service/data/US_historical/workbook.R')
